<!DOCTYPE html><html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" data-whc_version="25.0">
    <head><link rel="shortcut icon" href="../../../oxygen-webhelp/template/images/favicon.png"/><link rel="icon" href="../../../oxygen-webhelp/template/images/favicon.png"/><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/><meta name="viewport" content="width=device-width, initial-scale=1.0"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><meta name="copyright" content="(C) Copyright 2024"/><meta name="generator" content="DITA-OT"/><meta name="description" content="The basic tutorial creates a pipeline that reads a file from an HTTP resource URL, processes the data in two branches, and writes all data to a file system. You'll use data preview to help configure ..."/><meta name="prodname" content="Data Collector"/><meta name="version" content="3"/><meta name="release" content="16"/><meta name="modification" content="0"/>        
      <title>Basic Tutorial</title><!--  Generated with Oxygen version 25.1, build number 2023042410.  --><meta name="wh-path2root" content="../../../"/><meta name="wh-toc-id" content="concept_phf_cjt_ls-d16893e67578"/><meta name="wh-source-relpath" content="datacollector/UserGuide/Tutorial/BasicTutorial.dita"/><meta name="wh-out-relpath" content="datacollector/UserGuide/Tutorial/BasicTutorial.html"/>

    <link rel="stylesheet" type="text/css" href="../../../oxygen-webhelp/app/commons.css?buildId=2023042410"/>
    <link rel="stylesheet" type="text/css" href="../../../oxygen-webhelp/app/topic.css?buildId=2023042410"/>

    <script src="../../../oxygen-webhelp/app/options/properties.js?buildId=20240802104629"></script>
    <script src="../../../oxygen-webhelp/app/localization/strings.js?buildId=2023042410"></script>
    <script src="../../../oxygen-webhelp/app/search/index/keywords.js?buildId=20240802104629"></script>
    <script defer="defer" src="../../../oxygen-webhelp/app/commons.js?buildId=2023042410"></script>
    <script defer="defer" src="../../../oxygen-webhelp/app/topic.js?buildId=2023042410"></script>
<link rel="stylesheet" type="text/css" href="../../../oxygen-webhelp/template/light.css?buildId=2023042410"/><link rel="stylesheet" type="text/css" href="../../../skin.css"/></head>

    <body class="wh_topic_page frmBody">
        
        
        

        
<nav class="navbar navbar-default wh_header" data-whc_version="25.0">
    <div class="container-fluid">
        <div class="wh_header_flex_container navbar-nav navbar-expand-md navbar-dark">
            <div class="wh_logo_and_publication_title_container">
                <div class="wh_logo_and_publication_title">
                    
                    <!--
                            This component will be generated when the next parameters are specified in the transformation scenario:
                            'webhelp.logo.image' and 'webhelp.logo.image.target.url'.
                            See: http://oxygenxml.com/doc/versions/17.1/ug-editor/#topics/dita_webhelp_output.html.
                    -->
                    
                    <div class=" wh_publication_title "><a href="../../../index.html"><span class="booktitle">  <span class="ph mainbooktitle"><span class="ph">Data Collector</span> User Guide</span>  </span></a></div>
                    
                </div>
                
                <!-- The menu button for mobile devices is copied in the output only when the 'webhelp.show.top.menu' parameter is set to 'yes' -->
                
            </div>

            <div class="wh_top_menu_and_indexterms_link collapse navbar-collapse">
                
                
                <div class=" wh_indexterms_link "><a href="../../../indexTerms.html" title="Index" aria-label="Go to index terms page"><span>Index</span></a></div>
                
            </div>
        </div>
    </div>
</nav>

        <div class=" wh_search_input navbar-form wh_topic_page_search search " role="form">


<form id="searchForm" method="get" role="search" action="../../../search.html"><div><input type="search" placeholder="Search " class="wh_search_textfield" id="textToSearch" name="searchQuery" aria-label="Search query" required="required"/><button type="submit" class="wh_search_button" aria-label="Search"><span class="search_input_text">Search</span></button></div></form>

</div>
        
        <div class="container-fluid">
            <div class="row">

                <nav class="wh_tools d-print-none">
                    
<div data-tooltip-position="bottom" class=" wh_breadcrumb "><ol class="d-print-none"><li><span class="home"><a href="../../../index.html"><span>Home</span></a></span></li><li><div class="topicref" data-id="concept_nls_w1r_ks"><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/Tutorial-title.html">Tutorial</a></div></div></li><li class="active"><div class="topicref" data-id="concept_phf_cjt_ls"><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/BasicTutorial.html#concept_phf_cjt_ls">Basic Tutorial</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li></ol></div>



                    <div class="wh_right_tools "><button class="wh_hide_highlight" aria-label="Toggle search highlights" title="Toggle search highlights"></button><button class="webhelp_expand_collapse_sections" data-next-state="collapsed" aria-label="Collapse sections" title="Collapse sections"></button><div class=" wh_navigation_links "><span id="topic_navigation_links" class="navheader">
  
<span class="navprev"><a class="- topic/link link" href="../../../datacollector/UserGuide/Tutorial/BeforeYouBegin.html#concept_hg5_j2p_ms" title="Before You Begin" aria-label="Previous topic: Before You Begin" rel="prev"></a></span>  
<span class="navnext"><a class="- topic/link link" href="../../../datacollector/UserGuide/Tutorial/ExtendedTutorial.html#concept_w4n_gjt_ls" title="Extended Tutorial" aria-label="Next topic: Extended Tutorial" rel="next"></a></span>  </span></div>
<!--External resource link-->
<div class=" wh_print_link print d-none d-md-inline-block "><button onClick="window.print()" title="Print this page" aria-label="Print this page"></button></div>
                        
                        
                        
                        
                    </div>
                </nav>
            </div>

            

<div class="wh_content_area">
                <div class="row">
                    


                        <nav role="navigation" id="wh_publication_toc" class="col-lg-3 col-md-3 col-sm-12 d-md-block d-none d-print-none">
<div id="wh_publication_toc_content">


                            <div class=" wh_publication_toc " data-tooltip-position="right"><span class="expand-button-action-labels"><span id="button-expand-action" role="button" aria-label="Expand"></span><span id="button-collapse-action" role="button" aria-label="Collapse"></span><span id="button-pending-action" role="button" aria-label="Pending"></span></span><ul role="tree" aria-label="Table of Contents"><li role="treeitem" aria-expanded="false"><div data-tocid="concept_htw_ghg_jq-d16893e53" class="topicref" data-id="concept_htw_ghg_jq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_htw_ghg_jq-d16893e53-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Getting_Started/GettingStarted_Title.html#concept_htw_ghg_jq" id="concept_htw_ghg_jq-d16893e53-link">Getting Started</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_l2v_nlp_mpb-d16893e331" class="topicref" data-id="concept_l2v_nlp_mpb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_l2v_nlp_mpb-d16893e331-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/ReleaseNotes/ReleaseNotes.html#concept_l2v_nlp_mpb" id="concept_l2v_nlp_mpb-d16893e331-link">Release Notes</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_l4q_flb_kr-d16893e2582" class="topicref" data-id="concept_l4q_flb_kr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_l4q_flb_kr-d16893e2582-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Installation/Install_title.html" id="concept_l4q_flb_kr-d16893e2582-link">Installation</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ylh_yyz_ky-d16893e3984" class="topicref" data-id="concept_ylh_yyz_ky" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ylh_yyz_ky-d16893e3984-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Configuration/Config_title.html" id="concept_ylh_yyz_ky-d16893e3984-link">Configuration</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ejk_f1f_5v-d16893e7058" class="topicref" data-id="concept_ejk_f1f_5v" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ejk_f1f_5v-d16893e7058-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Upgrade/Upgrade_title.html" id="concept_ejk_f1f_5v-d16893e7058-link">Upgrade</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_qsw_cjy_bt-d16893e10103" class="topicref" data-id="concept_qsw_cjy_bt" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_qsw_cjy_bt-d16893e10103-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Pipeline_Design/PipelineDesign_title.html" id="concept_qsw_cjy_bt-d16893e10103-link">Pipeline Concepts and Design</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_qn1_wn4_kq-d16893e11199" class="topicref" data-id="concept_qn1_wn4_kq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_qn1_wn4_kq-d16893e11199-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Pipeline_Configuration/PipelineConfiguration_title.html" id="concept_qn1_wn4_kq-d16893e11199-link">Pipeline Configuration</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_hdr_gyw_41b-d16893e13057" class="topicref" data-id="concept_hdr_gyw_41b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_hdr_gyw_41b-d16893e13057-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Data_Formats/DataFormats-Title.html" id="concept_hdr_gyw_41b-d16893e13057-link">Data Formats</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_yjl_nc5_jq-d16893e14164" class="topicref" data-id="concept_yjl_nc5_jq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_yjl_nc5_jq-d16893e14164-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Origins/Origins_title.html" id="concept_yjl_nc5_jq-d16893e14164-link">Origins</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_yjl_nc5_jq-d16893e35197" class="topicref" data-id="concept_yjl_nc5_jq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_yjl_nc5_jq-d16893e35197-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Processors/Processors_title.html" id="concept_yjl_nc5_jq-d16893e35197-link">Processors</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_agj_cfj_br-d16893e44037" class="topicref" data-id="concept_agj_cfj_br" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_agj_cfj_br-d16893e44037-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Destinations/Destinations-title.html" id="concept_agj_cfj_br-d16893e44037-link">Destinations</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_umc_1lk_fx-d16893e56072" class="topicref" data-id="concept_umc_1lk_fx" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_umc_1lk_fx-d16893e56072-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Executors/Executors-title.html" id="concept_umc_1lk_fx-d16893e56072-link">Executors</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_xxd_f5r_kx-d16893e59696" class="topicref" data-id="concept_xxd_f5r_kx" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_xxd_f5r_kx-d16893e59696-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Event_Handling/EventFramework-Title.html#concept_xxd_f5r_kx" id="concept_xxd_f5r_kx-d16893e59696-link">Dataflow Triggers</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_zq5_pb4_flb-d16893e60134" class="topicref" data-id="concept_zq5_pb4_flb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_zq5_pb4_flb-d16893e60134-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Solutions/Solutions-title.html" id="concept_zq5_pb4_flb-d16893e60134-link">Solutions</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_ugp_kwf_xw-d16893e61337" class="topicref" data-id="concept_ugp_kwf_xw" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_ugp_kwf_xw-d16893e61337-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/DPM/DPM_title.html" id="concept_ugp_kwf_xw-d16893e61337-link">StreamSets Control Hub</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_fyf_gkq_4bb-d16893e62693" class="topicref" data-id="concept_fyf_gkq_4bb" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_fyf_gkq_4bb-d16893e62693-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Edge_Mode/EdgePipelines_title.html" id="concept_fyf_gkq_4bb-d16893e62693-link"><span class="ph">StreamSets Data Collector Edge</span></a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_wwq_gxc_py-d16893e63980" class="topicref" data-id="concept_wwq_gxc_py" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_wwq_gxc_py-d16893e63980-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Multithreaded_Pipelines/MultithreadedPipelines.html#concept_wwq_gxc_py" id="concept_wwq_gxc_py-d16893e63980-link">Multithreaded Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_gzw_tdm_p2b-d16893e64187" class="topicref" data-id="concept_gzw_tdm_p2b" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_gzw_tdm_p2b-d16893e64187-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Microservice/Microservice_Title.html#concept_gzw_tdm_p2b" id="concept_gzw_tdm_p2b-d16893e64187-link">Microservice Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="Orchestrators_Title-d16893e64348" class="topicref" data-id="Orchestrators_Title" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action Orchestrators_Title-d16893e64348-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Orchestration_Pipelines/OrchestrationPipelines_Title.html#Orchestrators_Title" id="Orchestrators_Title-d16893e64348-link">Orchestration Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_wr1_ktz_bt-d16893e64489" class="topicref" data-id="concept_wr1_ktz_bt" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_wr1_ktz_bt-d16893e64489-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/RPC_Pipelines/SDC_RPCpipelines_title.html#concept_wr1_ktz_bt" id="concept_wr1_ktz_bt-d16893e64489-link">SDC RPC Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_fpz_5r4_vs-d16893e64679" class="topicref" data-id="concept_fpz_5r4_vs" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_fpz_5r4_vs-d16893e64679-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Cluster_Mode/ClusterPipelines_title.html" id="concept_fpz_5r4_vs-d16893e64679-link">Cluster Pipelines</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_jjk_23z_sq-d16893e65172" class="topicref" data-id="concept_jjk_23z_sq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_jjk_23z_sq-d16893e65172-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Data_Preview/DataPreview_Title.html#concept_jjk_23z_sq" id="concept_jjk_23z_sq-d16893e65172-link">Data Preview</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_pgk_brx_rr-d16893e65458" class="topicref" data-id="concept_pgk_brx_rr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_pgk_brx_rr-d16893e65458-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Alerts/RulesAlerts_title.html#concept_pgk_brx_rr" id="concept_pgk_brx_rr-d16893e65458-link">Rules and Alerts</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_asx_fdz_sq-d16893e65960" class="topicref" data-id="concept_asx_fdz_sq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_asx_fdz_sq-d16893e65960-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Pipeline_Monitoring/PipelineMonitoring_title.html#concept_asx_fdz_sq" id="concept_asx_fdz_sq-d16893e65960-link">Pipeline Monitoring</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_o3l_dtr_5q-d16893e66304" class="topicref" data-id="concept_o3l_dtr_5q" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_o3l_dtr_5q-d16893e66304-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Pipeline_Maintenance/PipelineMaintenance_title.html#concept_o3l_dtr_5q" id="concept_o3l_dtr_5q-d16893e66304-link">Pipeline Maintenance</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_yms_ftm_sq-d16893e66768" class="topicref" data-id="concept_yms_ftm_sq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_yms_ftm_sq-d16893e66768-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Administration/Administration_title.html#concept_yms_ftm_sq" id="concept_yms_ftm_sq-d16893e66768-link">Administration</a></div></div></li><li role="treeitem" aria-expanded="true"><div data-tocid="concept_nls_w1r_ks-d16893e67508" class="topicref" data-id="concept_nls_w1r_ks" data-state="expanded"><span role="button" tabindex="0" aria-labelledby="button-collapse-action concept_nls_w1r_ks-d16893e67508-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/Tutorial-title.html" id="concept_nls_w1r_ks-d16893e67508-link">Tutorial</a></div></div><ul role="group" class="navbar-nav nav-list"><li role="treeitem"><div data-tocid="concept_is5_lrq_ks-d16893e67530" class="topicref" data-id="concept_is5_lrq_ks" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/Overview.html" id="concept_is5_lrq_ks-d16893e67530-link">Tutorial Overview</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem"><div data-tocid="concept_hg5_j2p_ms-d16893e67554" class="topicref" data-id="concept_hg5_j2p_ms" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/BeforeYouBegin.html#concept_hg5_j2p_ms" id="concept_hg5_j2p_ms-d16893e67554-link">Before You Begin</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="true" class="active"><div data-tocid="concept_phf_cjt_ls-d16893e67578" class="topicref" data-id="concept_phf_cjt_ls" data-state="expanded"><span role="button" tabindex="0" aria-labelledby="button-collapse-action concept_phf_cjt_ls-d16893e67578-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/BasicTutorial.html#concept_phf_cjt_ls" id="concept_phf_cjt_ls-d16893e67578-link">Basic Tutorial</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div><ul role="group" class="navbar-nav nav-list"><li role="treeitem"><div data-tocid="task_jmz_3dn_ls-d16893e67602" class="topicref" data-id="task_jmz_3dn_ls" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/BasicTutorial.html#task_jmz_3dn_ls" id="task_jmz_3dn_ls-d16893e67602-link">Create a Pipeline and Define Pipeline Properties</a><div class="wh-tooltip"><p class="shortdesc">When you configure a pipeline, you need to decide what to do with error records. You         can discard them or - more productively - write them to file, another pipeline, or a         destination, like Amazon S3 or Azure Event Hub. </p></div></div></div></li><li role="treeitem"><div data-tocid="task_ftt_2vq_ks-d16893e67627" class="topicref" data-id="task_ftt_2vq_ks" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/BasicTutorial.html#task_ftt_2vq_ks" id="task_ftt_2vq_ks-d16893e67627-link">Configure the Origin</a><div class="wh-tooltip"><p class="shortdesc">The origin represents the incoming data for the pipeline. When you configure the         origin, you define how to connect to the origin system, the type of data to be processed,         and other properties specific to the origin.</p></div></div></div></li><li role="treeitem"><div data-tocid="task_xb5_2sf_4s-d16893e67652" class="topicref" data-id="task_xb5_2sf_4s" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/BasicTutorial.html#task_xb5_2sf_4s" id="task_xb5_2sf_4s-d16893e67652-link">Preview Data </a><div class="wh-tooltip"><p class="shortdesc">To become more familiar with the data set and gather some important details for         pipeline configuration, let's preview the source data.</p></div></div></div></li><li role="treeitem"><div data-tocid="task_br5_rzq_ks-d16893e67677" class="topicref" data-id="task_br5_rzq_ks" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/BasicTutorial.html#task_br5_rzq_ks" id="task_br5_rzq_ks-d16893e67677-link">Route Data with the Stream Selector</a><div class="wh-tooltip"><p class="shortdesc">To route data to different streams for processing, we use the Stream Selector         processor.</p></div></div></div></li><li role="treeitem"><div data-tocid="task_cdx_kfm_ls-d16893e67702" class="topicref" data-id="task_cdx_kfm_ls" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/BasicTutorial.html#task_cdx_kfm_ls" id="task_cdx_kfm_ls-d16893e67702-link">Use Jython for Card Typing</a><div class="wh-tooltip"><p class="shortdesc"> Next, we'll evaluate credit card numbers to determine the credit card type. You can         use an Expression Evaluator processor to do the same calculations, but with a short script,         the Jython Evaluator processor is easier.</p></div></div></div></li><li role="treeitem"><div data-tocid="task_qbw_d3m_ls-d16893e67727" class="topicref" data-id="task_qbw_d3m_ls" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/BasicTutorial.html#task_qbw_d3m_ls" id="task_qbw_d3m_ls-d16893e67727-link">Mask Credit Card Numbers</a><div class="wh-tooltip"><p class="shortdesc"> Now let's prevent sensitive information from reaching internal databases by using a         Field Masker processor to mask the credit card numbers. </p></div></div></div></li><li role="treeitem"><div data-tocid="task_wsj_5tm_ls-d16893e67752" class="topicref" data-id="task_wsj_5tm_ls" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/BasicTutorial.html#task_wsj_5tm_ls" id="task_wsj_5tm_ls-d16893e67752-link">Write to the Destination</a><div class="wh-tooltip"><p class="shortdesc"><span class="ph">Data Collector</span>         can write data to many destinations. The Local FS destination writes to files in a local         file system. </p></div></div></div></li><li role="treeitem"><div data-tocid="task_wgb_t4p_ms-d16893e67779" class="topicref" data-id="task_wgb_t4p_ms" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/BasicTutorial.html#task_wgb_t4p_ms" id="task_wgb_t4p_ms-d16893e67779-link">Add a Corresponding Field with the Expression Evaluator</a><div class="wh-tooltip"><p class="shortdesc">The Jython Evaluator script added a new field to the credit payments branch. To         ensure all records have the same structure, we'll use the Expression Evaluator processor to         add the same field to the non-credit branch. </p></div></div></div></li><li role="treeitem"><div data-tocid="task_ets_dkj_ps-d16893e67804" class="topicref" data-id="task_ets_dkj_ps" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/BasicTutorial.html#task_ets_dkj_ps" id="task_ets_dkj_ps-d16893e67804-link">Create a Data Rule and Alert</a><div class="wh-tooltip"><p class="shortdesc">Now before we run the basic pipeline, let's add a data rule and alert. Data rules are         user-defined rules used to inspect data moving between two stages. They are a powerful way         to look for outliers and anomalous data. </p></div></div></div></li><li role="treeitem"><div data-tocid="task_m21_dyj_ps-d16893e67829" class="topicref" data-id="task_m21_dyj_ps" data-state="leaf"><span role="button" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/BasicTutorial.html#task_m21_dyj_ps" id="task_m21_dyj_ps-d16893e67829-link">Run the Basic Pipeline</a></div></div></li></ul></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_w4n_gjt_ls-d16893e67851" class="topicref" data-id="concept_w4n_gjt_ls" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_w4n_gjt_ls-d16893e67851-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Tutorial/ExtendedTutorial.html#concept_w4n_gjt_ls" id="concept_w4n_gjt_ls-d16893e67851-link">Extended Tutorial</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li></ul></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_sh3_frm_tq-d16893e68001" class="topicref" data-id="concept_sh3_frm_tq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_sh3_frm_tq-d16893e68001-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Troubleshooting/Troubleshooting_title.html#concept_sh3_frm_tq" id="concept_sh3_frm_tq-d16893e68001-link">Troubleshooting</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_xbx_rs1_tq-d16893e68798" class="topicref" data-id="concept_xbx_rs1_tq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_xbx_rs1_tq-d16893e68798-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Glossary/Glossary_title.html#concept_xbx_rs1_tq" id="concept_xbx_rs1_tq-d16893e68798-link">Glossary</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_jn1_nzb_kv-d16893e68843" class="topicref" data-id="concept_jn1_nzb_kv" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_jn1_nzb_kv-d16893e68843-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Apx-DataFormats/DataFormat_Title.html#concept_jn1_nzb_kv" id="concept_jn1_nzb_kv-d16893e68843-link">Data Formats by Stage</a><div class="wh-tooltip"><p class="shortdesc"></p></div></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_pvm_yt3_wq-d16893e68958" class="topicref" data-id="concept_pvm_yt3_wq" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_pvm_yt3_wq-d16893e68958-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Expression_Language/ExpressionLanguage_title.html" id="concept_pvm_yt3_wq-d16893e68958-link">Expression Language</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_vcj_1ws_js-d16893e69669" class="topicref" data-id="concept_vcj_1ws_js" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_vcj_1ws_js-d16893e69669-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Apx-RegEx/RegEx-Title.html#concept_vcj_1ws_js" id="concept_vcj_1ws_js-d16893e69669-link">Regular Expressions</a></div></div></li><li role="treeitem" aria-expanded="false"><div data-tocid="concept_chv_vmj_wr-d16893e69787" class="topicref" data-id="concept_chv_vmj_wr" data-state="not-ready"><span role="button" tabindex="0" aria-labelledby="button-expand-action concept_chv_vmj_wr-d16893e69787-link" class="wh-expand-btn"></span><div class="title"><a href="../../../datacollector/UserGuide/Apx-GrokPatterns/GrokPatterns_title.html#concept_chv_vmj_wr" id="concept_chv_vmj_wr-d16893e69787-link">Grok Patterns</a></div></div></li></ul></div>
                        

</div>
</nav>
                    


                    
                    <div id="wh_topic_body" class="col-lg-7 col-md-9 col-sm-12">
<button id="wh_close_publication_toc_button" class="close-toc-button d-none" aria-label="Toggle publishing table of content" aria-controls="wh_publication_toc" aria-expanded="true"><span class="close-toc-icon-container"><span class="close-toc-icon"></span></span></button><button id="wh_close_topic_toc_button" class="close-toc-button d-none" aria-label="Toggle topic table of content" aria-controls="wh_topic_toc" aria-expanded="true"><span class="close-toc-icon-container"><span class="close-toc-icon"></span></span></button>

                        
<div class=" wh_topic_content body "><main role="main"><article class="" role="article" aria-labelledby="ariaid-title1"><article class="nested0" aria-labelledby="ariaid-title1" id="concept_phf_cjt_ls">
 <h1 class="- topic/title title topictitle1" id="ariaid-title1">Basic Tutorial</h1>
 
 <div class="- topic/body concept/conbody body conbody"><p class="- topic/shortdesc shortdesc"></p>
  <p class="- topic/p p">The basic tutorial creates a pipeline that reads a file from an HTTP resource URL, processes
   the data in two branches, and writes all data to a file system. You'll use data preview to help
   configure the pipeline, then create a data alert and run the pipeline. </p>
  <div class="- topic/p p">Here are high level steps for building and running the basic pipeline:<ol class="- topic/ol ol" id="concept_phf_cjt_ls__ol_emy_rhp_ms" data-ofbid="concept_phf_cjt_ls__ol_emy_rhp_ms">
    <li class="- topic/li li">Configure pipeline properties, primarily error handling.</li>
    <li class="- topic/li li">Add an HTTP Client origin to represent the data to be processed. </li>
    <li class="- topic/li li">Preview source data to determine the field-level details needed for the pipeline.</li>
    <li class="- topic/li li">Use a Stream Selector processor to route credit card transactions to the primary branch and
     cash transactions to the secondary branch. We'll define a required field to discard records
     without a payment type.</li>
    <li class="- topic/li li">Configure a Jython Evaluator processor to perform custom processing that determines the
     credit card type based on the credit card number. </li>
    <li class="- topic/li li">Add a Field Masker processor to mask credit card numbers. Use a required field to discard
     records without credit card numbers.</li>
    <li class="- topic/li li">Connect both branches to a Local FS destination.</li>
    <li class="- topic/li li">In the secondary branch, use an Expression Evaluator processor to add fields to the cash
     records to match the credit card records. Use data preview to verify the fields to add. </li>
    <li class="- topic/li li">Add a data rule to raise an alert if too many credit card payments are missing credit card
     numbers.</li>
    <li class="- topic/li li">Start the pipeline and monitor the results.</li>
   </ol></div>
 </div>
<article class="- topic/topic task/task topic task nested1" aria-labelledby="ariaid-title2" id="task_jmz_3dn_ls">
    <h2 class="- topic/title title topictitle2" id="ariaid-title2">Create a Pipeline and Define Pipeline Properties</h2>
    
    <div class="- topic/body task/taskbody body taskbody"><p class="- topic/shortdesc shortdesc">When you configure a pipeline, you need to decide what to do with error records. You
        can discard them or - more productively - write them to file, another pipeline, or a
        destination, like Amazon S3 or Azure Event Hub. </p>
        <section class="- topic/section task/context section context">
            <p class="- topic/p p">Write error records to one of these locations as a convenient way to review and
                reprocess with error records without having to stop the pipeline. </p>
            <p class="- topic/p p">This tutorial writes the records to a local file. </p>
        </section>
        <section class="- topic/ol task/steps ol steps"><ol class="- topic/ol task/steps ol steps" id="task_jmz_3dn_ls__steps_hsz_b2n_ls" data-ofbid="task_jmz_3dn_ls__steps_hsz_b2n_ls"><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">If necessary, log in to <span class="- topic/ph ph">Control Hub</span>. </span>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd"><span class="- topic/ph ph">In the top toolbar, click <span class="+ topic/ph ui-d/menucascade ph menucascade"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">Quick Start</span><abbr title="and then"> &gt; </abbr><span class="+ topic/ph ui-d/uicontrol ph uicontrol">Create a pipeline</span></span>.</span></span>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Enter the following name: <kbd class="+ topic/ph sw-d/userinput ph userinput">Tutorial</kbd>.</span>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Use the defaults to create a blank <span class="- topic/ph ph">Data Collector</span> pipeline, and then
                    click <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Next</span>.</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <p class="- topic/p p">By default, <span class="- topic/ph ph">Control Hub</span>
                        selects an accessible authoring <span class="- topic/ph ph">Data Collector</span> that you have
                        read permission on and that has the most recent reported time. To select
                        another <span class="- topic/ph ph">Data Collector</span>, click
                            <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Click here to select</span>.</p>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Click <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Save &amp; Open in Canvas</span>.</span>
                <div class="- topic/itemgroup task/info itemgroup info"><p class="- topic/p p">A blank pipeline opens in the canvas.</p><img class="- topic/image image" id="task_jmz_3dn_ls__image_g34_jt2_jsb" src="../Graphics/Tutorial-Canvas-platform.png"/></div>
                <div class="- topic/itemgroup task/info itemgroup info">Note a few helpful areas: <div class="table-container"><table class="- topic/table table frame-all" id="task_jmz_3dn_ls__table_ezq_5ys_lt" data-ofbid="task_jmz_3dn_ls__table_ezq_5ys_lt" data-cols="3"><caption></caption><colgroup><col style="width:9.090909090909092%"/><col style="width:18.181818181818183%"/><col style="width:72.72727272727273%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__1">Number</th>
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__2">Name</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__3">Description</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__1">1</td>
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__2">Pipeline Creation Help Bar </td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__3">Indicates that the origin is missing and offers a list of
                                        origins to choose from. <p class="- topic/p p">If the Pipeline Creation Help Bar
                                            does not display, you might have disabled it. To enable
                                            it, in the top right corner of the <span class="- topic/ph ph">Data Collector</span> window, click the <span class="+ topic/ph ui-d/menucascade ph menucascade"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">Help</span><abbr title="and then"> &gt; </abbr><span class="+ topic/ph ui-d/uicontrol ph uicontrol">Settings</span></span>. Clear the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Hide Pipeline
                                                Creation Help Bar</span> option.</p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__1">2</td>
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__2">Issues icon</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__3">Click to display the list of pipeline issues found by
                                        implicit validation.</td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__1">3</td>
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__2">Stage library panel</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__3">Displays a list of available stages by default.</td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__1">4</td>
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__2">Stage menu</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__3">Changes the stages that display in the stage library.
                                    </td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__1">5</td>
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__2">Properties panel / Preview panel / Monitor panel </td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__3">When you configure a pipeline, the Properties panel
                                        displays the properties of the pipeline or selected stage.
                                        You can resize, minimize and maximize the panel. <p class="- topic/p p">When you
                                            preview data, the Preview panel displays the input and
                                            output data for the selected stage or group of stages.
                                            </p><p class="- topic/p p">When you monitor a running pipeline, the Monitor
                                            panel displays real-time metrics and statistics.
                                            </p><div class="- topic/note note note note_note"><span class="note__title">Note:</span> Some icons and options might not display in
                                            the UI. The items that display are based on the task
                                            that you are performing and roles assigned to your user
                                            account. </div></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__1"><img class="- topic/image image" id="task_jmz_3dn_ls__image_jbb_gym_jsb" src="../Graphics/Icon_StageControl.png" height="16" width="21"/></td>
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__2">Stage library icon</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__3">Toggles the display of the stage library panel.</td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__1"><img class="- topic/image image" id="task_jmz_3dn_ls__image_gsq_1tp_fsb" src="../Graphics/icon_ConfigStageLibrary_Origin.png" height="22" width="21"/></td>
                                    <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__2">Stage control icon</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_jmz_3dn_ls__table_ezq_5ys_lt__entry__3">Controls the stages that display in the stage library
                                        panel. In the image above, the panel displays only origins.
                                        This icon changes depending on the selection. The default
                                        icon (<img class="- topic/image image" id="task_jmz_3dn_ls__image_ndm_3vn_jsb" src="../Graphics/icon_ConfigStageLibrary_Platform.png" height="20" width="20"/>) indicates all types
                                        of stages.</td>
                                </tr>
                            </tbody></table></div></div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">In the Properties panel, click the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Error Records</span> tab. And
                    for the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Error Records</span> property, select <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Write
                        to File</span>.</span>
                <div class="- topic/itemgroup task/info itemgroup info">This writes error records to a file so you can deal with error records without
                    having to stop the pipeline.</div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Click the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Error Records - Write to File</span> tab and configure
                    the following properties.</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <p class="- topic/p p">Use the defaults for properties that aren't listed:</p>
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_jmz_3dn_ls__table_zvy_kfn_ls" data-ofbid="task_jmz_3dn_ls__table_zvy_kfn_ls" data-cols="2"><caption></caption><colgroup><col style="width:27.47252747252747%"/><col style="width:72.52747252747253%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_jmz_3dn_ls__table_zvy_kfn_ls__entry__1">Write to File Property</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_jmz_3dn_ls__table_zvy_kfn_ls__entry__2">Description</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_jmz_3dn_ls__table_zvy_kfn_ls__entry__1">Directory</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_jmz_3dn_ls__table_zvy_kfn_ls__entry__2">Directory for error record files. Enter the directory
                                        that you set up for the tutorial. We recommended:
                                            <pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>/&lt;base directory&gt;/tutorial/error</code></pre><div class="- topic/note note note note_note"><span class="note__title">Note:</span> To
                                            prevent validation errors, the directory must already
                                            exist.</div></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_jmz_3dn_ls__table_zvy_kfn_ls__entry__1">Files Prefix</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_jmz_3dn_ls__table_zvy_kfn_ls__entry__2">This defines a prefix for error record files. <p class="- topic/p p">By default, the files are prefixed with "SDC" and an expression that
                  returns the <span class="- topic/ph ph">Data Collector</span> ID, but
                  that's more than we need here. </p><p class="- topic/p p">Delete the default and enter the following prefix:
                                                <kbd class="+ topic/ph sw-d/userinput ph userinput">err_</kbd>.</p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_jmz_3dn_ls__table_zvy_kfn_ls__entry__1">Max File Size</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_jmz_3dn_ls__table_zvy_kfn_ls__entry__2">For the tutorial, reduce the size of the files to
                                        something more manageable, such as <kbd class="+ topic/ph sw-d/userinput ph userinput">5</kbd>
                                        or <kbd class="+ topic/ph sw-d/userinput ph userinput">1</kbd> MB.</td>
                                </tr>
                            </tbody></table></div>
                </div>
            </li></ol></section>
        <section class="- topic/section task/result section result">
            <p class="- topic/p p">Now we'll start building the pipeline...</p>
        </section>
    </div>
</article><article class="- topic/topic task/task topic task nested1" aria-labelledby="ariaid-title3" id="task_ftt_2vq_ks">
    <h2 class="- topic/title title topictitle2" id="ariaid-title3">Configure the Origin</h2>
    
    <div class="- topic/body task/taskbody body taskbody"><p class="- topic/shortdesc shortdesc">The origin represents the incoming data for the pipeline. When you configure the
        origin, you define how to connect to the origin system, the type of data to be processed,
        and other properties specific to the origin.</p>
        <section class="- topic/section task/context section context" id="task_ftt_2vq_ks__context_kg1_fym_vrb" data-ofbid="task_ftt_2vq_ks__context_kg1_fym_vrb">
            <p class="- topic/p p">This tutorial builds a pipeline that reads a sample CSV file from an HTTP resource
                URL, processes the data to convert the data type of several fields, and then writes
                the data to a JSON file on your local machine.</p>
            <p class="- topic/p p"> The sample CSV file includes some invalid data, so you'll also see how <span class="- topic/ph ph">StreamSets</span> handles errors when you preview the pipeline. </p>
        </section>
        <section class="- topic/ol task/steps ol steps"><ol class="- topic/ol task/steps ol steps" id="task_ftt_2vq_ks__steps_lg1_fym_vrb" data-ofbid="task_ftt_2vq_ks__steps_lg1_fym_vrb"><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">From the pipeline creation help bar, click <span class="+ topic/ph ui-d/menucascade ph menucascade"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">Select Origin</span><abbr title="and then"> &gt; </abbr><span class="+ topic/ph ui-d/uicontrol ph uicontrol">HTTP Client</span></span>, as follows:</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <p class="- topic/p p"><img class="- topic/image image" id="task_ftt_2vq_ks__image_wkz_ryy_fsb" src="../Graphics/Try_PipelineOrigin.png" height="144" width="568"/></p>
                    <p class="- topic/p p">The origin is added to the canvas.</p>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">In the properties panel below the canvas, click the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">HTTP</span>
                    tab.</span>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Configure the HTTP properties as follows:</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_ftt_2vq_ks__table_wv3_1cr_l4b" data-ofbid="task_ftt_2vq_ks__table_wv3_1cr_l4b" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_ftt_2vq_ks__table_wv3_1cr_l4b__entry__1">HTTP Property</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_ftt_2vq_ks__table_wv3_1cr_l4b__entry__2">Value</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_ftt_2vq_ks__table_wv3_1cr_l4b__entry__1">Resource URL</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_ftt_2vq_ks__table_wv3_1cr_l4b__entry__2"><code class="+ topic/ph pr-d/codeph ph codeph">https://docs.streamsets.com/datacollector/sample_data/tutorial/nyc_taxi_data.csv</code>
                                    </td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_ftt_2vq_ks__table_wv3_1cr_l4b__entry__1">Mode</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_ftt_2vq_ks__table_wv3_1cr_l4b__entry__2"><kbd class="+ topic/ph sw-d/userinput ph userinput">Polling</kbd></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_ftt_2vq_ks__table_wv3_1cr_l4b__entry__1">Polling Interval</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_ftt_2vq_ks__table_wv3_1cr_l4b__entry__2"><kbd class="+ topic/ph sw-d/userinput ph userinput">600000</kbd></td>
                                </tr>
                            </tbody></table></div>
                    <div class="- topic/p p">
                        <div class="- topic/note note note note_note"><span class="note__title">Note:</span> In most cases you would use batch mode to configure the origin to read
                            a single file. In batch mode, the origin processes all available data
                            and then stops the pipeline and job. However, when the origin uses batch
                            mode to read a small amount of data, the <span class="- topic/ph ph">Data Collector</span> engine runs and stops the pipeline before you have a chance to
                            monitor the data in real time. Setting the mode to polling with a 10
                            minute (or 600000 millisecond) interval causes the origin to read the
                            full contents of the file, wait 10 minutes, and then read the contents
                            of the file again.</div>
                    </div>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Use the default values for the remaining properties.</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <p class="- topic/p p">The <span class="+ topic/keyword ui-d/wintitle keyword wintitle">HTTP</span> tab should be configured as follows:</p>
                    <p class="- topic/p p"><img class="- topic/image image" id="task_ftt_2vq_ks__image_xh3_zyy_fsb" src="../Graphics/Try_HTTPClient_HTTPTab.png" height="426" width="781"/></p>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Click the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Data Format</span> tab.</span>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Configure the data format as follows:</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_ftt_2vq_ks__table_ayh_dbr_l4b" data-ofbid="task_ftt_2vq_ks__table_ayh_dbr_l4b" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_ftt_2vq_ks__table_ayh_dbr_l4b__entry__1">Data Format Property</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_ftt_2vq_ks__table_ayh_dbr_l4b__entry__2">Value</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_ftt_2vq_ks__table_ayh_dbr_l4b__entry__1">Data Format</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_ftt_2vq_ks__table_ayh_dbr_l4b__entry__2"><kbd class="+ topic/ph sw-d/userinput ph userinput">Delimited</kbd></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_ftt_2vq_ks__table_ayh_dbr_l4b__entry__1">Header Line</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_ftt_2vq_ks__table_ayh_dbr_l4b__entry__2"><kbd class="+ topic/ph sw-d/userinput ph userinput">With Header Line</kbd></td>
                                </tr>
                            </tbody></table></div>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Use the default values for the remaining properties.</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <p class="- topic/p p">Since the sample data is read from a file, the origin reads all fields as
                        strings.</p>
                </div>
            </li></ol></section>
    </div>
</article><article class="- topic/topic task/task topic task nested1" aria-labelledby="ariaid-title4" id="task_xb5_2sf_4s">
    <h2 class="- topic/title title topictitle2" id="ariaid-title4">Preview Data </h2>
    
    <div class="- topic/body task/taskbody body taskbody"><p class="- topic/shortdesc shortdesc">To become more familiar with the data set and gather some important details for
        pipeline configuration, let's preview the source data.</p>
        <section class="- topic/section task/context section context">
            <div class="- topic/p p">Here are some key details that we need to configure the pipeline: <ul class="- topic/ul ul" id="task_xb5_2sf_4s__ul_a41_hvh_4s" data-ofbid="task_xb5_2sf_4s__ul_a41_hvh_4s">
                    <li class="- topic/li li">The field that contains payment information - We'll use this to route data
                        in the Stream Selector processor. </li>
                    <li class="- topic/li li">The field that contains credit card numbers - We'll use this to mask the
                        data in the Field Masker processor.</li>
                </ul></div>
            <p class="- topic/p p">When you access data in a field, you specify the field path for the field. The field
                path depends on the complexity of the record: <code class="+ topic/ph pr-d/codeph ph codeph">/&lt;fieldname&gt;</code> for
                simple records and <code class="+ topic/ph pr-d/codeph ph codeph">&lt;path to field&gt;/&lt;fieldname&gt;</code> for more
                complex records.  </p>
            <p class="- topic/p p">Because we're using the List-Map root field type, we can use
                    <code class="+ topic/ph pr-d/codeph ph codeph">/&lt;fieldname&gt;</code>. </p>
            <p class="- topic/p p">To start data preview, all stages must be connected and all required properties
                defined - though not necessarily correctly. Since the origin is configured and the
                only stage, the pipeline should be ready to preview as is. </p>
        </section>
        <section class="- topic/ol task/steps ol steps"><ol class="- topic/ol task/steps ol steps" id="task_xb5_2sf_4s__steps_jxv_fpf_4s" data-ofbid="task_xb5_2sf_4s__steps_jxv_fpf_4s"><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Above the pipeline canvas, click the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Preview</span> icon:
                        <img class="- topic/image image" id="task_xb5_2sf_4s__image_x4k_njp_rs" src="../Graphics/icon_Preview.png" height="13" width="17"/>.</span>
                <div class="- topic/itemgroup task/info itemgroup info">If the icon is not enabled, make sure you completed all the previous steps for
                    the tutorial. If more than one issue displays in the list, correct any issue
                    except Validation_0011. <p class="- topic/p p">Validation_0011 indicates that the HTTP Client origin
                        isn't connected to anything yet and does not prevent data
                    preview.</p></div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">In the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Preview Configuration</span> dialog box, configure the
                    following properties. </span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <p class="- topic/p p">Use the defaults for properties that aren't listed:</p>
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_xb5_2sf_4s__table_z2x_325_lt" data-ofbid="task_xb5_2sf_4s__table_z2x_325_lt" data-cols="2"><caption></caption><colgroup><col style="width:30%"/><col style="width:70%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_xb5_2sf_4s__table_z2x_325_lt__entry__1">Data Preview Property</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_xb5_2sf_4s__table_z2x_325_lt__entry__2">Description</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_xb5_2sf_4s__table_z2x_325_lt__entry__1">Preview Source</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_xb5_2sf_4s__table_z2x_325_lt__entry__2">Use the default <kbd class="+ topic/ph sw-d/userinput ph userinput">Configured Source</kbd>
                                        to use the sample source data.</td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_xb5_2sf_4s__table_z2x_325_lt__entry__1">Write to Destinations and Executors</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_xb5_2sf_4s__table_z2x_325_lt__entry__2">By default, this property is not selected. In general,
                                        you should keep this property clear to avoid writing data to
                                        destination systems or triggering tasks in destination
                                        systems.</td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_xb5_2sf_4s__table_z2x_325_lt__entry__1">Show Field Type</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_xb5_2sf_4s__table_z2x_325_lt__entry__2">By default, this property is selected. Keep it selected
                                        to see the data type of fields in the record. </td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_xb5_2sf_4s__table_z2x_325_lt__entry__1">Remember the Configuration</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_xb5_2sf_4s__table_z2x_325_lt__entry__2">Select to use these properties each time you run data
                                        preview. <p class="- topic/p p">When you select this option, the UI enters data
                                            preview without showing this dialog box again.
                                        </p></td>
                                </tr>
                            </tbody></table></div>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Click <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Run Preview</span>.</span>
                <div class="- topic/itemgroup task/stepresult itemgroup stepresult">The Preview Panel displays a list of 10 output records for HTTP Client
                    origin in list view. If you drill down a bit, you see that each record displays
                    an ordered list of field names and values with the data type of each field.
                        <p class="- topic/p p">Because the data is read from a file, all fields are
                            strings:</p><p class="- topic/p p"><img class="- topic/image image" id="task_xb5_2sf_4s__image_w4q_xmv_vrb" src="../Graphics/Tutorial-PreviewFirst_Platform.png"/></p><p class="- topic/p p">Note that in the Preview panel, the
                        preview records display on the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Records</span> tab: <img class="- topic/image image" id="task_xb5_2sf_4s__image_czm_lsl_zt" src="../Graphics/icon_PrevRecords.png" height="16" width="19"/>. To review or change properties, use the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Stage
                            Configuration</span> tab: <img class="- topic/image image" id="task_xb5_2sf_4s__image_gf2_kx1_tt" src="../Graphics/icon_PrevStageConfig.png" height="12" width="15"/>. To change preview properties, use the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Preview
                            Configuration</span> tab: <img class="- topic/image image" id="task_xb5_2sf_4s__image_a3l_mx1_tt" src="../Graphics/icon_PrevPreviewConfig.png" height="16" width="19"/>.</p></div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">For a full view of the data, click the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Table View</span> icon:
                        <img class="- topic/image image" id="task_xb5_2sf_4s__image_zxq_1zf_4s" src="../Graphics/icon_PrevTableView.png" height="12" width="16"/>.</span>
                <div class="- topic/itemgroup task/info itemgroup info">The Preview panel displays the first several columns of data. <p class="- topic/p p"><img class="- topic/image image" id="task_xb5_2sf_4s__image_cwz_mnv_vrb" src="../Graphics/Tutorial-PreviewDirectory_Platform.png"/></p></div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">To view all columns, click <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Show All</span>.</span>
                <div class="- topic/itemgroup task/info itemgroup info">If you scroll to the right, you can verify that the
                        <code class="+ topic/ph pr-d/codeph ph codeph">credit_card</code> field contains credit card numbers for the
                    first three records. When you scroll back to the <code class="+ topic/ph pr-d/codeph ph codeph">payment_type</code>
                    field, notice this corresponds to records where the payment type is
                        <code class="+ topic/ph pr-d/codeph ph codeph">CRD</code>.</div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Now that we have the information that we need, click the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Close
                        Preview</span> icon: <img class="- topic/image image" id="task_xb5_2sf_4s__image_zs2_2bg_4s" src="../Graphics/icon_PrevClose.png" height="13" width="16"/>.</span>
            </li></ol></section>
        <section class="- topic/section task/result section result">Now we know the following: <ul class="- topic/ul ul" id="task_xb5_2sf_4s__ul_glg_yj1_mt" data-ofbid="task_xb5_2sf_4s__ul_glg_yj1_mt">
                <li class="- topic/li li">Payment type information is in the <code class="+ topic/ph pr-d/codeph ph codeph">payment_type</code> field and
                    credit card information is in the  <code class="+ topic/ph pr-d/codeph ph codeph">credit_card</code> field.</li>
                <li class="- topic/li li">To use these fields in expressions, we'll use their field paths:
                        <code class="+ topic/ph pr-d/codeph ph codeph">/payment_type</code> and <code class="+ topic/ph pr-d/codeph ph codeph">/credit_card</code>. </li>
                <li class="- topic/li li">To route records paid by credit card, we'll look for records where the payment
                    type is <code class="+ topic/ph pr-d/codeph ph codeph">CRD</code>. </li>
            </ul></section>
    </div>
</article><article class="- topic/topic task/task topic task nested1" aria-labelledby="ariaid-title5" id="task_br5_rzq_ks">
    <h2 class="- topic/title title topictitle2" id="ariaid-title5">Route Data with the Stream Selector</h2>
    
    <div class="- topic/body task/taskbody body taskbody"><p class="- topic/shortdesc shortdesc">To route data to different streams for processing, we use the Stream Selector
        processor.</p>
        <section class="- topic/section task/context section context"><p class="- topic/p p">The Stream Selector processor routes data to different streams based on
                user-defined conditions. Any data not captured by a user-defined condition routes to
                the default stream.</p><p class="- topic/p p">We'll route credit card transactions to a credit card
                stream for processing. All other transactions will go to the default
                stream.</p><p class="- topic/p p">We'll also define a required field to drop records with no payment
                type. When you define a required field, a record must include data for the specified
                field to enter the stage. Records that don't include data in required fields are
                sent to the pipeline for error handling. If you configured the pipeline to write to
                file, that's where error records go. </p>To represent data in a field, we use the
                <code class="+ topic/ph pr-d/codeph ph codeph">record:value</code> function. This returns field values associated with
            the field.<div class="- topic/p p">To capture records with credit card payments, use the following condition:
                <pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>${record:value('/payment_type') == 'CRD'}</code></pre></div><p class="- topic/p p">Note that we
                enclose expressions in a dollar sign and curly brackets. You can use single or
                double quotation marks around strings. For more information about the expression
                language, see <a class="- topic/xref xref" href="../Expression_Language/ExpressionLanguage_overview.html#concept_p54_4kl_vq">Expression Language</a>.</p></section>
        <section class="- topic/ol task/steps ol steps"><ol class="- topic/ol task/steps ol steps" id="task_br5_rzq_ks__steps_xhx_3rk_ls" data-ofbid="task_br5_rzq_ks__steps_xhx_3rk_ls"><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">From the Pipeline Creation Help Bar, click <span class="+ topic/ph ui-d/menucascade ph menucascade"><span class="+ topic/ph ui-d/uicontrol ph uicontrol">Select Processor to Connect</span><abbr title="and then"> &gt; </abbr><span class="+ topic/ph ui-d/uicontrol ph uicontrol">Stream Selector</span></span>. Or, in the Stage Library panel, select the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Stream
                        Selector</span> processor (<img class="- topic/image image" id="task_br5_rzq_ks__image_rt1_3m1_mt" src="../Graphics/Tutorial_SSelector_icon.png" height="20" width="20"/>) and connect the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">HTTP Client</span> origin to
                    it. </span>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">On the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">General</span> tab, click in the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Required
                        Fields</span> text box. </span>
                <div class="- topic/itemgroup task/info itemgroup info">A list of available fields displays because you already performed data
                    preview. It also displays when the pipeline is valid for data preview. </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">To discard records with no payment type information, click <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Advanced
                        Options </span>and select the following field:
                        <code class="+ topic/ph pr-d/codeph ph codeph">/payment_type</code>.</span>
                <div class="- topic/itemgroup task/info itemgroup info">If a list does not appear, you can manually enter the field path:
                        <code class="+ topic/ph pr-d/codeph ph codeph">/payment_type</code>.</div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">To configure the Stream Selector processor condition, click the
                        <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Conditions</span> tab.</span>
                <div class="- topic/itemgroup task/info itemgroup info">A condition for the default stream displays. The default stream represents any
                    records not captured by other conditions.</div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Click the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Add</span> icon: <img class="- topic/image image" id="task_br5_rzq_ks__image_snw_l5p_15" src="../Graphics/icon_ConfigAddIcon.png" height="15" width="15"/>.</span>
                <div class="- topic/itemgroup task/info itemgroup info">A condition text box displays in the Property panel, and the corresponding
                    output location appears on the stage in the canvas.</div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">The following condition captures records where a credit card is used for
                    payment. You can copy and paste the expression, but try typing it in to see how
                    the expression completion feature helps you select the function and ensure valid
                    syntax.</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>${record:value('/payment_type') == 'CRD'}</code></pre>
                </div>
                <div class="- topic/itemgroup task/info itemgroup info">All records that match this condition pass to the first output stream. All
                    other records are captured by the default condition and passed through the
                    second output stream.</div>
            </li></ol></section>
        <section class="- topic/section task/result section result">The Stream Selector processor configuration should look like this:<p class="- topic/p p"><img class="- topic/image image" id="task_br5_rzq_ks__image_xbh_3mm_jsb" src="../Graphics/Tutorial-StreamSelector-Platform.png" height="372" width="1103"/></p></section>
    </div>
</article><article class="- topic/topic task/task topic task nested1" aria-labelledby="ariaid-title6" id="task_cdx_kfm_ls">
    <h2 class="- topic/title title topictitle2" id="ariaid-title6">Use Jython for Card Typing</h2>
    
    <div class="- topic/body task/taskbody body taskbody"><p class="- topic/shortdesc shortdesc"> Next, we'll evaluate credit card numbers to determine the credit card type. You can
        use an Expression Evaluator processor to do the same calculations, but with a short script,
        the Jython Evaluator processor is easier.</p>
        <section class="- topic/section task/context section context">
            <p class="- topic/p p">You can use custom scripts with the JavaScript Evaluator and the Jython Evaluator
                processors to perform processing that is not easily performed using other <span class="- topic/ph ph">Data Collector</span>
                processors. When using scripts to handle list-map data, the script must treat the
                data as maps.</p>
            <p class="- topic/p p">The Jython script that we provide creates an additional field,
                    <code class="+ topic/ph pr-d/codeph ph codeph">credit_card_type</code>, and generates the credit card type by
                evaluating the first few digits of the credit card number. The script returns an
                error message if the record has a credit card payment type without a corresponding
                credit card number. </p>
        </section>
        <section class="- topic/ol task/steps ol steps"><ol class="- topic/ol task/steps ol steps" id="task_cdx_kfm_ls__steps_lnv_lfm_ls" data-ofbid="task_cdx_kfm_ls__steps_lnv_lfm_ls"><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Add a <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Jython Evaluator</span> processor to the canvas.</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <p class="- topic/p p">If the Jython Evaluator processor isnât listed in the stage library, youâll
                        need to install the Jython stage library first.</p>
                </div>
                <ol type="a" class="- topic/ol task/substeps ol substeps" id="task_cdx_kfm_ls__substeps_yhd_f2z_1bb" data-ofbid="task_cdx_kfm_ls__substeps_yhd_f2z_1bb">
                    <li class="- topic/li task/substep li substep substepexpand">
                        <span class="- topic/ph task/cmd ph cmd">Select the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Jython Evaluator</span> processor in the
                            library stage panel.</span>
                    </li>
                    <li class="- topic/li task/substep li substep substepexpand">
                        <span class="- topic/ph task/cmd ph cmd">In the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Missing Stage Libraries</span> window, click
                                <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Update Deployment</span> to confirm your
                            choice.</span>
                        <div class="- topic/itemgroup task/info itemgroup info"><span class="- topic/ph ph">Control Hub</span>Â installs the missing stage library. </div>
                    </li>
                    <li class="- topic/li task/substep li substep substepexpand">
                        <span class="- topic/ph task/cmd ph cmd">Click <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Restart Engine</span>. The Missing Stage
                            Libraries window closes and the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Restart Data
                                Collector</span> window displays.</span>
                    </li>
                    <li class="- topic/li task/substep li substep substepexpand">
                        <span class="- topic/ph task/cmd ph cmd">Click <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Restart Data Collector</span>.</span>
                    </li>
                    <li class="- topic/li task/substep li substep substepexpand">
                        <span class="- topic/ph task/cmd ph cmd">In the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Restart Data Collector</span> window, click
                                <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Close</span> to exit.</span>
                    </li>
                    <li class="- topic/li task/substep li substep substepexpand">
                        <span class="- topic/ph task/cmd ph cmd">Add the Jython Evaluator processor to the canvas.</span>
                    </li>
                </ol>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Connect the first output location of the Stream Selector processor to the
                    Jython Evaluator processor.</span>
                <div class="- topic/itemgroup task/info itemgroup info">This routes records paid by credit card to the Jython Evaluator processor.
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">With the Jython Evaluator processor selected, in the Properties panel, click
                    the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Jython</span> tab. </span>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">In the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Advanced</span> tab, use the default <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Batch by
                        Batch</span> record processing mode to process data in batches, instead
                    of record by record. </span>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">In the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Script</span> text box, review the information in the
                    comments, then delete it. Paste in the following script: </span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>try: 
  for record in records:
    cc = record.value['credit_card']
    if cc == '':
      error.write(record, "Payment type was CRD, but credit card was null")
      continue

    cc_type = ''
    if cc.startswith('4'):
      cc_type = 'Visa'
    elif cc.startswith(('51','52','53','54','55')):
      cc_type = 'MasterCard'
    elif cc.startswith(('34','37')):
      cc_type = 'AMEX'
    elif cc.startswith(('300','301','302','303','304','305','36','38')):
      cc_type = 'Diners Club'
    elif cc.startswith(('6011','65')):
      cc_type = 'Discover'
    elif cc.startswith(('2131','1800','35')):
      cc_type = 'JCB'
    else:
      cc_type = 'Other'

    record.value['credit_card_type'] = cc_type

    output.write(record)
except Exception as e:
  error.write(record, e.message)</code></pre>
                    <div class="- topic/p p">
                        <div class="- topic/note note note note_note"><span class="note__title">Note:</span> Incorrect indentation can cause Jython validation errors. For best
                            results, copy the script from the online help. Copying the script from
                            the PDF can result in incorrect indentation.<p class="- topic/p p">To launch
                                context-sensitive help, click the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Help</span> icon
                                in the Properties panel. Then in the table of contents, scroll down
                                to find the Data Collector Tutorial chapter &gt; Basic Tutorial &gt; Use
                                Jython for Card Typing. </p></div>
                    </div>
                </div>
            </li></ol></section>
        <section class="- topic/section task/result section result">In the Jython Evaluator, the script should look like this:<p class="- topic/p p"><img class="- topic/image image" id="task_cdx_kfm_ls__image_jt1_4ch_4s" src="../Graphics/Tutorial-Jython.png" height="406" width="490"/></p></section>
    </div>
</article><article class="- topic/topic task/task topic task nested1" aria-labelledby="ariaid-title7" id="task_qbw_d3m_ls">
    <h2 class="- topic/title title topictitle2" id="ariaid-title7">Mask Credit Card Numbers</h2>
    
    <div class="- topic/body task/taskbody body taskbody"><p class="- topic/shortdesc shortdesc"> Now let's prevent sensitive information from reaching internal databases by using a
        Field Masker processor to mask the credit card numbers. </p>
        <section class="- topic/section task/context section context">
            <p class="- topic/p p">The Field Masker processor provides fixed and variable-length masks to mask all data
                in a field. To reveal specified positions in the data, you can use custom masking.
                To reveal a group of positions within the data, you can use a regular expression
                mask to define the structure of the data and then reveal one or more groups.</p>
            <div class="- topic/p p">For the credit card numbers, we'll use the following regular expression to mask all
                but the last four digits:<pre class="+ topic/pre pr-d/codeblock pre codeblock"><code>(.*)([0-9]{4})</code></pre></div>
            <p class="- topic/p p">The regular expression defines two groups so we can reveal the second group.</p>
        </section>
        <section class="- topic/ol task/steps ol steps"><ol class="- topic/ol task/steps ol steps" id="task_qbw_d3m_ls__steps_wwy_fjm_ls" data-ofbid="task_qbw_d3m_ls__steps_wwy_fjm_ls"><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Add a <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Field Masker</span> processor to the canvas and connect
                    the Jython Evaluator processor to it. </span>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">In the Properties panel, click the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Mask</span> tab.</span>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Click in the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Fields to Mask</span> field and enter the field
                        <kbd class="+ topic/ph sw-d/userinput ph userinput">/credit_card</kbd>. </span>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">To use a regular expression as a mask and display the last 4 digits of the
                    credit card number, configure the rest of the properties as follows:</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_qbw_d3m_ls__table_ejl_msm_ls" data-ofbid="task_qbw_d3m_ls__table_ejl_msm_ls" data-cols="2"><caption></caption><colgroup><col style="width:28.49002849002849%"/><col style="width:71.5099715099715%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_qbw_d3m_ls__table_ejl_msm_ls__entry__1">Field Masker Property</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_qbw_d3m_ls__table_ejl_msm_ls__entry__2">Value</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_qbw_d3m_ls__table_ejl_msm_ls__entry__1">Mask Type</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_qbw_d3m_ls__table_ejl_msm_ls__entry__2"><kbd class="+ topic/ph sw-d/userinput ph userinput">Regular Expression</kbd></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_qbw_d3m_ls__table_ejl_msm_ls__entry__1">Regular Expression</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_qbw_d3m_ls__table_ejl_msm_ls__entry__2">
                                        <code class="+ topic/ph pr-d/codeph ph codeph">(.*)([0-9]{4})</code></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_qbw_d3m_ls__table_ejl_msm_ls__entry__1">Groups to Show</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_qbw_d3m_ls__table_ejl_msm_ls__entry__2"><kbd class="+ topic/ph sw-d/userinput ph userinput">2</kbd></td>
                                </tr>
                            </tbody></table></div>
                </div>
            </li></ol></section>
        <section class="- topic/section task/result section result">Here's how the Field Masker processor should look in the pipeline:<p class="- topic/p p"><img class="- topic/image image" id="task_qbw_d3m_ls__image_gdz_xnn_jsb" src="../Graphics/Tutorial_FieldMasker_Platform.png" height="469" width="866"/></p></section>
    </div>
</article><article class="- topic/topic task/task topic task nested1" aria-labelledby="ariaid-title8" id="task_wsj_5tm_ls">
    <h2 class="- topic/title title topictitle2" id="ariaid-title8">Write to the Destination</h2>
    
    <div class="- topic/body task/taskbody body taskbody"><p class="- topic/shortdesc shortdesc"><span class="- topic/ph ph">Data Collector</span>
        can write data to many destinations. The Local FS destination writes to files in a local
        file system. </p>
        <section class="- topic/section task/context section context">
            <p class="- topic/p p">When you configure the Local FS destination, you define the directory template. This
                determines the naming convention for the output directories that are created. </p>
        </section>
        <section class="- topic/ol task/steps ol steps"><ol class="- topic/ol task/steps ol steps" id="task_wsj_5tm_ls__steps_xdd_wwm_ls" data-ofbid="task_wsj_5tm_ls__steps_xdd_wwm_ls"><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Add a <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Local FS</span> destination to the canvas and connect the
                        <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Field Masker</span> processor to it.</span>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Click the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Output Files</span> tab, click <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Show Advanced
                        Properties</span>, and then configure the following properties.</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <p class="- topic/p p">Use the defaults for properties that aren't listed:</p>
                    <div class="table-container"><table class="- topic/table table frame-all" id="task_wsj_5tm_ls__table_kcy_5bn_ls" data-ofbid="task_wsj_5tm_ls__table_kcy_5bn_ls" data-cols="2"><caption></caption><colgroup><col style="width:29.49852507374631%"/><col style="width:70.50147492625369%"/></colgroup><thead class="- topic/thead thead">
                                <tr class="- topic/row">
                                    <th class="- topic/entry entry colsep-1 rowsep-1" id="task_wsj_5tm_ls__table_kcy_5bn_ls__entry__1">Local FS Property</th>
                                    <th class="- topic/entry entry colsep-0 rowsep-1" id="task_wsj_5tm_ls__table_kcy_5bn_ls__entry__2">Configuration</th>
                                </tr>
                            </thead><tbody class="- topic/tbody tbody">
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_wsj_5tm_ls__table_kcy_5bn_ls__entry__1">Directory Template</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_wsj_5tm_ls__table_kcy_5bn_ls__entry__2">By default, the directory template includes datetime
                                        variables to create a directory structure for output files.
                                        This is intended for writing large volumes of data. <p class="- topic/p p">Since
                                            we only have the sample file to process, we don't need
                                            the datetime variables. Go ahead and delete the default
                                            and enter the directory where you want the files to be
                                            written. </p><p class="- topic/p p">We suggest: <code class="+ topic/ph pr-d/codeph ph codeph">/&lt;base
                                                directory&gt;/tutorial/destination</code>.</p></td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-1" headers="task_wsj_5tm_ls__table_kcy_5bn_ls__entry__1">Files Prefix</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-1" headers="task_wsj_5tm_ls__table_kcy_5bn_ls__entry__2">Defines a prefix for output file names.<p class="- topic/p p">By default, the files are prefixed with "SDC" and an expression that
                  returns the <span class="- topic/ph ph">Data Collector</span> ID, but
                  that's more than we need here. </p>Let's simplify and use <kbd class="+ topic/ph sw-d/userinput ph userinput">out_</kbd>
                                        instead.</td>
                                </tr>
                                <tr class="- topic/row">
                                    <td class="- topic/entry entry colsep-1 rowsep-0" headers="task_wsj_5tm_ls__table_kcy_5bn_ls__entry__1">Max File Size (MB)</td>
                                    <td class="- topic/entry entry colsep-0 rowsep-0" headers="task_wsj_5tm_ls__table_kcy_5bn_ls__entry__2">For the tutorial, let's lower the file size to something
                                        manageable, like <kbd class="+ topic/ph sw-d/userinput ph userinput">5</kbd> or
                                            <kbd class="+ topic/ph sw-d/userinput ph userinput">1</kbd>.</td>
                                </tr>
                            </tbody></table></div>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Click the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Data Format</span> tab, and configure the following
                    properties. </span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <p class="- topic/p p">Use the defaults for properties that aren't listed:</p>
                    <div class="simpletable-container"><table class="- topic/simpletable simpletable frame-all" id="task_wsj_5tm_ls__simpletable_jyj_2vq_ps" data-ofbid="task_wsj_5tm_ls__simpletable_jyj_2vq_ps"><colgroup><col style="width:50%"/><col style="width:50%"/></colgroup><thead><tr class="- topic/sthead sthead">
                            <th class="- topic/stentry stentry" scope="col" id="task_wsj_5tm_ls__simpletable_jyj_2vq_ps__stentry__1">Delimited Property</th>
                            <th class="- topic/stentry stentry" scope="col" id="task_wsj_5tm_ls__simpletable_jyj_2vq_ps__stentry__2">Value</th>
                        </tr></thead><tbody><tr class="- topic/strow strow">
                            <td class="- topic/stentry stentry" headers="task_wsj_5tm_ls__simpletable_jyj_2vq_ps__stentry__1">Data Format</td>
                            <td class="- topic/stentry stentry" headers="task_wsj_5tm_ls__simpletable_jyj_2vq_ps__stentry__2"><kbd class="+ topic/ph sw-d/userinput ph userinput">Delimited</kbd></td>
                        </tr><tr class="- topic/strow strow">
                            <td class="- topic/stentry stentry" headers="task_wsj_5tm_ls__simpletable_jyj_2vq_ps__stentry__1">Header Line</td>
                            <td class="- topic/stentry stentry" headers="task_wsj_5tm_ls__simpletable_jyj_2vq_ps__stentry__2"><kbd class="+ topic/ph sw-d/userinput ph userinput">With Header Line</kbd></td>
                        </tr></tbody></table></div>
                </div>
            </li></ol></section>
        <section class="- topic/section task/result section result">This completes the primary branch:<p class="- topic/p p"><img class="- topic/image image" id="task_wsj_5tm_ls__image_ucy_vpn_jsb" src="../Graphics/Tutorial_LocalFS_Platform.png" height="96" width="802"/></p><p class="- topic/p p"> Now, we'll go back to the Stream Selector
                processor and complete the secondary branch.</p></section>
    </div>
</article><article class="- topic/topic task/task topic task nested1" aria-labelledby="ariaid-title9" id="task_wgb_t4p_ms">
    <h2 class="- topic/title title topictitle2" id="ariaid-title9">Add a Corresponding Field with the Expression Evaluator</h2>
    
    <div class="- topic/body task/taskbody body taskbody"><p class="- topic/shortdesc shortdesc">The Jython Evaluator script added a new field to the credit payments branch. To
        ensure all records have the same structure, we'll use the Expression Evaluator processor to
        add the same field to the non-credit branch. </p>
        <section class="- topic/section task/context section context">
            <p class="- topic/p p">This ensures that all records have the same format when written to the
                destination.</p>
            <p class="- topic/p p">To do this, let's use data preview to verify how the Jython Evaluator processor adds
                the credit card type to records.</p>
        </section>
        <section class="- topic/ol task/steps ol steps"><ol class="- topic/ol task/steps ol steps" id="task_wgb_t4p_ms__steps_mr4_2pp_ms" data-ofbid="task_wgb_t4p_ms__steps_mr4_2pp_ms"><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Click the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Preview</span> icon: <img class="- topic/image image" id="task_wgb_t4p_ms__image_yyt_nfx_fsb" src="../Graphics/icon_Preview.png"/>.</span>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">In the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Preview Configuration</span> window, click <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Run
                        Preview</span>. </span>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">In the pipeline, click the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Jython Evaluator</span> processor to
                    view its output.</span>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Expand the first output record, scroll down, and notice the new field
                    highlighted in green: <code class="+ topic/ph pr-d/codeph ph codeph">/credit_card_type</code>.</span>
                <div class="- topic/itemgroup task/info itemgroup info"><img class="- topic/image image" id="task_wgb_t4p_ms__image_jxk_4nm_jsb" src="../Graphics/Tutorial-Exp-JythonPreview_Platform.png" height="816" width="1469"/></div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Click <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Close Preview</span>.</span>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Add an <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Expression Evaluator</span> processor to the canvas and
                    connect the second, default stream of the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Stream Selector</span>
                    processor to it.</span>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Click the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Expressions</span> tab.</span>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Configure the following <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Field Expression</span>
                    properties:</span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <div class="simpletable-container"><table class="- topic/simpletable simpletable frame-all" id="task_wgb_t4p_ms__simpletable_xdm_rcr_ps" data-ofbid="task_wgb_t4p_ms__simpletable_xdm_rcr_ps"><colgroup><col style="width:50%"/><col style="width:50%"/></colgroup><thead><tr class="- topic/sthead sthead">
                            <th class="- topic/stentry stentry" scope="col" id="task_wgb_t4p_ms__simpletable_xdm_rcr_ps__stentry__1">Output Field</th>
                            <th class="- topic/stentry stentry" scope="col" id="task_wgb_t4p_ms__simpletable_xdm_rcr_ps__stentry__2">Value</th>
                        </tr></thead><tbody><tr class="- topic/strow strow">
                            <td class="- topic/stentry stentry" headers="task_wgb_t4p_ms__simpletable_xdm_rcr_ps__stentry__1"><code class="+ topic/ph pr-d/codeph ph codeph">/credit_card_type</code></td>
                            <td class="- topic/stentry stentry" headers="task_wgb_t4p_ms__simpletable_xdm_rcr_ps__stentry__2"><kbd class="+ topic/ph sw-d/userinput ph userinput">n/a</kbd></td>
                        </tr></tbody></table></div>
                </div>
                <div class="- topic/itemgroup task/info itemgroup info">This creates a <code class="+ topic/ph pr-d/codeph ph codeph">credit_card_type</code> field that indicates the
                    information is not applicable.</div>
                <div class="- topic/itemgroup task/info itemgroup info">Since we're using <code class="+ topic/ph pr-d/codeph ph codeph">n/a</code> as a constant for the expression, we
                    don't need to use the dollar sign and brackets for the expression. But if we
                    wanted to use them, we could define them as
                        <code class="+ topic/ph pr-d/codeph ph codeph">${'credit_card_type'}</code> and <code class="+ topic/ph pr-d/codeph ph codeph">${'N/A'}</code>. </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Link the Expression Evaluator processor to the Local FS destination. </span>
                <div class="- topic/itemgroup task/info itemgroup info">This streams the data from this branch to the destination, merging data from
                    both branches: <p class="- topic/p p"><img class="- topic/image image" id="task_wgb_t4p_ms__image_evm_krn_jsb" src="../Graphics/Tutorial-Expression-Platform.png" height="535" width="1093"/></p></div>
            </li></ol></section>
    </div>
</article><article class="- topic/topic task/task topic task nested1" aria-labelledby="ariaid-title10" id="task_ets_dkj_ps">
    <h2 class="- topic/title title topictitle2" id="ariaid-title10">Create a Data Rule and Alert</h2>
    
    <div class="- topic/body task/taskbody body taskbody"><p class="- topic/shortdesc shortdesc">Now before we run the basic pipeline, let's add a data rule and alert. Data rules are
        user-defined rules used to inspect data moving between two stages. They are a powerful way
        to look for outliers and anomalous data. </p>
        <section class="- topic/section task/context section context">
            <p class="- topic/p p">Data rules and alerts require a detailed understanding of the data passing through
                the pipeline. For more general pipeline monitoring information, you can use metric
                rules and alerts.</p>
            <p class="- topic/p p">The script in the Jython Evaluator processor creates error records for credit card
                transactions without credit card numbers. We can create a data rule and alert to let
                us know when the record count reaches a specified threshold. </p>
            <p class="- topic/p p">We'll use an expression with the <code class="+ topic/ph pr-d/codeph ph codeph">record:value()</code> function to identify
                when the credit card number field, <code class="+ topic/ph pr-d/codeph ph codeph">/credit_card</code>, is null. The
                function returns the data in the specified field. </p>
        </section>
        <section class="- topic/ol task/steps ol steps"><ol class="- topic/ol task/steps ol steps" id="task_ets_dkj_ps__steps_cw5_qqj_ps" data-ofbid="task_ets_dkj_ps__steps_cw5_qqj_ps"><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Between the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Stream Selector</span> and the <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Jython
                        Evaluator</span> processors, select the link or Data Inspection icon:
                        <img class="- topic/image image" id="task_ets_dkj_ps__image_w5t_2w2_jsb" src="../Graphics/Icon_DataAlert.png" height="15" width="14"/>.</span>
                <div class="- topic/itemgroup task/info itemgroup info">The <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Data Rules</span> tab displays in the Preview panel. </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Click <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Add</span>.</span>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">In the <span class="+ topic/keyword ui-d/wintitle keyword wintitle">Data Rule</span> dialog box, configure the following
                    properties. </span>
                <div class="- topic/itemgroup task/info itemgroup info">
                    <p class="- topic/p p">Use the defaults for properties that aren't listed:</p>
                    <div class="simpletable-container"><table class="- topic/simpletable simpletable frame-all" id="task_ets_dkj_ps__simpletable_z34_ctj_ps" data-ofbid="task_ets_dkj_ps__simpletable_z34_ctj_ps"><colgroup><col style="width:30.3951367781155%"/><col style="width:69.6048632218845%"/></colgroup><thead><tr class="- topic/sthead sthead">
                            <th class="- topic/stentry stentry" scope="col" id="task_ets_dkj_ps__simpletable_z34_ctj_ps__stentry__1">Data Rule Property</th>
                            <th class="- topic/stentry stentry" scope="col" id="task_ets_dkj_ps__simpletable_z34_ctj_ps__stentry__2">Value</th>
                        </tr></thead><tbody><tr class="- topic/strow strow">
                            <td class="- topic/stentry stentry" headers="task_ets_dkj_ps__simpletable_z34_ctj_ps__stentry__1">Label</td>
                            <td class="- topic/stentry stentry" headers="task_ets_dkj_ps__simpletable_z34_ctj_ps__stentry__2"><kbd class="+ topic/ph sw-d/userinput ph userinput">Missing Card Numbers</kbd></td>
                        </tr><tr class="- topic/strow strow">
                            <td class="- topic/stentry stentry" headers="task_ets_dkj_ps__simpletable_z34_ctj_ps__stentry__1">Condition</td>
                            <td class="- topic/stentry stentry" headers="task_ets_dkj_ps__simpletable_z34_ctj_ps__stentry__2"><kbd class="+ topic/ph sw-d/userinput ph userinput">${record:value("/credit_card") ==
                                ""}</kbd></td>
                        </tr><tr class="- topic/strow strow">
                            <td class="- topic/stentry stentry" headers="task_ets_dkj_ps__simpletable_z34_ctj_ps__stentry__1">Sampling Percentage</td>
                            <td class="- topic/stentry stentry" headers="task_ets_dkj_ps__simpletable_z34_ctj_ps__stentry__2"><kbd class="+ topic/ph sw-d/userinput ph userinput">35</kbd></td>
                        </tr><tr class="- topic/strow strow">
                            <td class="- topic/stentry stentry" headers="task_ets_dkj_ps__simpletable_z34_ctj_ps__stentry__1">Alert Text</td>
                            <td class="- topic/stentry stentry" headers="task_ets_dkj_ps__simpletable_z34_ctj_ps__stentry__2"><kbd class="+ topic/ph sw-d/userinput ph userinput">At least 10 missing credit card
                                numbers!</kbd></td>
                        </tr><tr class="- topic/strow strow">
                            <td class="- topic/stentry stentry" headers="task_ets_dkj_ps__simpletable_z34_ctj_ps__stentry__1">Threshold Value</td>
                            <td class="- topic/stentry stentry" headers="task_ets_dkj_ps__simpletable_z34_ctj_ps__stentry__2"><kbd class="+ topic/ph sw-d/userinput ph userinput">10</kbd></td>
                        </tr></tbody></table></div>
                    <p class="- topic/p p">This creates an alert that is triggered after finding ten records that don't
                        include credit card numbers in the credit card payment stream.</p>
                    <div class="- topic/note note note note_note"><span class="note__title">Note:</span> With a larger data set, a smaller sampling percentage and higher threshold
                        value might be appropriate, but we'll use these numbers for the purposes of
                        the tutorial.</div>
                </div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">Click <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Save</span>.</span>
                <div class="- topic/itemgroup task/info itemgroup info">The rule displays in the data rule list. And the Data Inspection icon darkens
                    to show that a data rule is configured for the stream.</div>
            </li><li class="- topic/li task/step li step">
                <span class="- topic/ph task/cmd ph cmd">To enable the data rule and alert, click <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Enable</span>. </span>
                <div class="- topic/itemgroup task/info itemgroup info">Notice the Data Inspection icon becomes a darker grey to show that a data rule
                    is active on the stream.</div>
            </li></ol></section>
    </div>
</article><article class="- topic/topic task/task topic task nested1" aria-labelledby="ariaid-title11" id="task_m21_dyj_ps">
    <h2 class="- topic/title title topictitle2" id="ariaid-title11">Run the Basic Pipeline</h2>
    <div class="- topic/body task/taskbody body taskbody">
        <section class="- topic/section task/context section context">Now that the basic pipeline is complete, you can perform a test run of the draft
            pipeline in the pipeline canvas. A test run allows you to quickly test the pipeline
            logic without having to publish the pipeline. To perform the test, click <span class="+ topic/ph ui-d/uicontrol ph uicontrol">Test
                Run &gt; Start Pipeline</span>.<p class="- topic/p p">The UI enters Monitor mode and shows summary
                statistics in the Monitor panel. At some point as the pipeline runs, the data alert
                triggers and the location of the triggered data alert turns red.</p><p class="- topic/p p"></p>Click the
            Data Alert icon to view the data alert notification. Then, close the notification and
            explore the information available in the Monitor panel.<p class="- topic/p p"><img class="- topic/image image" id="task_m21_dyj_ps__image_ifb_3w2_jsb" src="../Graphics/Tutorial-RunPipeline_Platform.png"/>
            </p><p class="- topic/p p">Note that when you select an unused part of the canvas, the Monitor panel
                displays monitoring information for the entire pipeline. When you select a stage, it
                displays information for the stage. </p><p class="- topic/p p">The Jython Evaluator processor shows 40
                error records. Click the error records number to see the list of cached error
                records and related error messages. </p><p class="- topic/p p">You can also select the red Data
                Inspection icon to view information about the data alert and view the error records
                associated with the data alert.</p><img class="- topic/image image" id="task_m21_dyj_ps__image_a5c_kqn_jsb" src="../Graphics/Tutorial_RunningPipeline_DataAlert.png" height="711" width="1012"/><p class="- topic/p p">In the Extended Tutorial, we'll add more stages and see
                more data transformations. We can use data preview to step through the pipeline to
                review how each stage processes the data.</p><p class="- topic/p p"> </p></section>
    </div>
</article></article></article></main></div>

                        
                        
                        


                    </div>
                    
                </div>
            </div>


        </div> <nav class="navbar navbar-default wh_footer" data-whc_version="25.0">
  <div class=" footer-container  mx-auto">
    <!-- script for Data Collector, all flavors, but only used when accessed directly, not from portal --><script>
  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

  ga('create', 'UA-60917135-3', 'auto');
  ga('send', 'pageview');
</script>
  </div>
</nav>

        
        <div id="go2top">
            <span class="oxy-icon oxy-icon-up"></span>
        </div>
        
        <!-- The modal container for images -->
        <div id="modal_img_large" class="modal">
            <span class="close oxy-icon oxy-icon-remove"></span>
            <!-- Modal Content (The Image) -->
            <div id="modal_img_container"></div>
            <!-- Modal Caption (Image Text) -->
            <div id="caption"></div>
        </div>
        
        
        Â© 2023 StreamSets, Inc.

    </body>
</html>